package org.clockwise.driver;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import org.clockwise.feature.AccountPool;
import org.clockwise.feature.QueryContext;
import org.clockwise.multimethod.ListNet;
import org.clockwise.multimethod.PageRank;
import org.clockwise.multimethod.SRW;
import org.clockwise.srw.AccountRelationPool;
import org.clockwise.srw.RefGraph;
import org.clockwise.srw.UserRelationPool;
import org.clockwise.util.MyFileReader;
import org.clockwise.util.MyFileWriter;
import org.clockwise.util.MyMath;

public class LinkerDriver {

	public static String DATA_DIR = "D:\\Workspace\\WeiboData\\";

	// public static HashMap<Long, Account> userMap = null;
	// public static AccountPool accountPool = null;
	// public static List<QueryContext> contextList = null;

	public static List<List<List<String>>> verbose = null;

	public static void main(String[] args) {
		long begin = System.currentTimeMillis();
		// CorpusExtractionDriver.run();
		// GlobalTFIDF.computeTFIDF();
		// GlobalTFIDF.computeUserTFIDF();
		// System.out.println(TestNLPIR.test("呵呵\n"));
		// CorpusExtractionDriver.searchPrev();
		// T2SConvertor.convertFileT2S(DATA_DIR + "info_resorted");
		// AccountOperationDriver.recordLabelAccount();
		// AccountOperationDriver.testAccount();

		// HashMap<Long, Account> map = AccountOperationDriver.getAccount();
		// long subbegin = System.currentTimeMillis();
		// AccountPool ap = new AccountPool(map);
		// long subend = System.currentTimeMillis();
		// System.out.println("AccountPool is ready in " + (subend - subbegin) +
		// "ms");
		// List<QueryContext> contextList =
		// CorpusExtractionDriver.getQueryContextFromText(ap);
		// subbegin = System.currentTimeMillis();
		// List<QueryContext> contextList =
		// CorpusExtractionDriver.getQueryContext(ap);
		// subend = System.currentTimeMillis();
		// System.out.println("QueryContext list is ready in " + (subend -
		// subbegin) + "ms");
		// CorpusExtractionDriver.saveQueryContextList(contextList);
		// for (int i = 2; i < 10; i++) {
		// test(i, contextList);
		// }
		// srw(contextList, ap);

		/* I hope the structure would not change. */
//		 CorpusExtractionDriver.generateQueryDataFile();

		// PageRank.test(data, dict);

		if (args.length == 1) {
			DATA_DIR = args[0];
		}
		
//		 verbose = CorpusExtractionDriver.getVerbose();
		// crossValidation(5);

		// for (double a = 0.1; a < 1; a++) {
		// PageRank.alpha = a;
		// System.out.println("alpha = " + a);
		// SupervisedRandomWalks();
		// crossValidation(2, false);
		// }
//		testOverflow();
//		testInv();
//		SupervisedRandomWalks();
//		UserRelationPool.generateRelationFile();
		crossValidation(1, true);

		// rediscoverCorpus();
		
		testParameters();
		
		long end = System.currentTimeMillis();
		System.out
				.println("Overall time consumption: " + (end - begin) + " ms");

	}
	
	public static void testInv() {
		double[][] m = {
				{1, 2, 3},
				{2, 5, 8},
				{0, 2, 1},
		};
		MyMath.printMatrix(SRW.inv(m));
	}
	
	public static void testOverflow() {
		double c = 1.0;
		for (int i = 0; i < 10; i++) {
			c *= 10;
			System.out.println(i + "\t" + c + "\t" + Math.exp(c) + "\t" + (Math.exp(c) / Math.exp(c)));
		}
		c = -1.0;
		for (int i = 0; i < 10; i++) {
			c *= 10;
			System.out.println(i + "\t" + c + "\t" + Math.exp(c));
		}
	}

	public static void rediscoverCorpus() {
		MyFileReader mfr = new MyFileReader(LinkerDriver.DATA_DIR
				+ "cleanDoc.txt");
		String temp = null;

		List<String> corpus = new ArrayList<String>();
		List<String> segments = new ArrayList<String>();

		StringBuilder sb = new StringBuilder();
		while ((temp = mfr.getNextLine()) != null) {
			if (temp.startsWith("<doc>")) {
				sb = new StringBuilder();
			} else if (temp.startsWith("</doc>")) {
				corpus.add(sb.toString());
			} else {
				sb.append(temp);
			}
		}
		mfr.close();
		System.out.println(corpus.size());

		mfr = new MyFileReader(LinkerDriver.DATA_DIR + "SelectedSegment.txt");
		while ((temp = mfr.getNextLine()) != null) {
			if (temp.length() > 3) {
				segments.add(temp.trim());
			}
		}
		System.out.println(segments.size());
		mfr.close();

		MyFileWriter mfw = new MyFileWriter(LinkerDriver.DATA_DIR
				+ "WholePage.txt", true);
		for (int i = 0; i < segments.size(); i++) {
			String target = segments.get(i);
			mfw.write(target + "\n");
			Iterator<String> iter = corpus.iterator();
			List<String> possible = new ArrayList<String>();
			while (iter.hasNext()) {
				String whole = iter.next();
				if (whole.contains(target)) {
					possible.add(whole);
				}
			}
			mfw.write("<doc>\n");
			mfw.write(possible.size() + "\n");
			for (int j = 0; j < possible.size(); j++) {
				mfw.write("\t" + possible.get(j) + "\n");
			}
			mfw.write("\n");
			mfw.write("</doc>\n");
		}
		mfw.close();
	}

	public static boolean isRecord = false;
	public static void recordError(int contextId, int queryId, int predictId,
			int expectedSize, String file) {
		if (isRecord) {
			if (verbose == null) {
				verbose = CorpusExtractionDriver.getVerbose();
			}
			// System.out.println("ContextId = " + contextId);
			assert (contextId < verbose.size()) : verbose.size();

			List<List<String>> qc = verbose.get(contextId);

//			System.out.println("QueryId = " + queryId);
			assert (queryId < qc.size()) : qc.size();
			List<String> q = qc.get(queryId);

			// System.out.println("predictId = " + predictId);
			// System.out.println("Expected size = " + expectedSize);
			// System.out.println("Actual potential size = " + q.size());
			// assert (expectedSize == q.size()) : q.size();
			assert (predictId < q.size()) : q.size();

			MyFileWriter mfr = new MyFileWriter(file, false);
			mfr.write("Context" + contextId + "\t" + "Query" + queryId + "\n");
			mfr.write("\tCorrect: " + q.get(0) + "\n");
			mfr.write("\tPredict: " + q.get(predictId) + "\n");

			mfr.write("\n");
			mfr.close();

			// System.out.println("\tcorrect: " + q.potential.get(0).toString());
			// System.out.println("\tpredict: " +
			// q.potential.get(predictId).toString());
		}
		

	}
	
	public static void recordError(int contextId, int queryId, int predictId, 
			int expectedSize, double[] cf, double[] wf, String file) {
		if (isRecord) {
			if (verbose == null) {
				verbose = CorpusExtractionDriver.getVerbose();
			}
			// System.out.println("ContextId = " + contextId);
			assert (contextId < verbose.size()) : verbose.size();

			List<List<String>> qc = verbose.get(contextId);

//			System.out.println("QueryId = " + queryId);
			assert (queryId < qc.size()) : qc.size();
			List<String> q = qc.get(queryId);

			// System.out.println("predictId = " + predictId);
			// System.out.println("Expected size = " + expectedSize);
			// System.out.println("Actual potential size = " + q.size());
			// assert (expectedSize == q.size()) : q.size();
			assert (predictId < q.size()) : q.size();

			MyFileWriter mfr = new MyFileWriter(file, false);
			mfr.write("Context" + contextId + "\t" + "Query" + queryId + "\n");
			mfr.write("\tCorrect: " + q.get(0) + "\n");
			
			mfr.write("\n");
			mfr.write("\tPredict: " + q.get(predictId) + "\n");
			
			mfr.write("\n");
			mfr.close();

			// System.out.println("\tcorrect: " + q.potential.get(0).toString());
			// System.out.println("\tpredict: " +
			// q.potential.get(predictId).toString());
		}
		

	}

	// public static void initCorpusData() {
	// System.out.println("Initializing corpus data...");
	// LinkerDriver.userMap = AccountOperationDriver.getAccount();
	// LinkerDriver.accountPool = new AccountPool(LinkerDriver.userMap);
	// LinkerDriver.contextList = CorpusExtractionDriver
	// .getQueryContext(LinkerDriver.accountPool);
	// System.out.println("Initialize corpus data done");
	// }
	
	public static MyFileWriter writer = null;
	public static void testParameters() {
		writer = new MyFileWriter("result_11.txt", false);
//		double alpha = 0.1;
		double alpha = 0.15;
		double beta = 100;
		double lamda = 6;
		double eta = 10;
		double oRate = 5E-12;
		double tRate = 0.0002;
//		for (lamda = 6; lamda < 10; lamda *= 1.3) {
//			for (alpha = 0.3; alpha <= 0.3; alpha += 0.05) {
//				for (beta = 120; beta < 121; beta *= 2) {
					RealSRW(alpha, beta, lamda, eta, oRate, tRate);
//				}
//			}
//		}
	}

	public static void RealSRW(double alpha, double beta, double lamda,
			double eta, double oRate, double tRate) {
		List<List<List<double[]>>> data = new ArrayList<List<List<double[]>>>();
		List<List<List<Long>>> dict = new ArrayList<List<List<Long>>>();
		List<double[][]> relation = new ArrayList<double[][]>();
		List<double[][]> ucFeature = new ArrayList<double[][]>();
		List<double[][][]> uuFeature = new ArrayList<double[][][]>();
		List<int[]> ruler = new ArrayList<int[]>();
		List<Long[]> idList = new ArrayList<Long[]>();
		
		int all = DataReader.readData(data, dict);
//		AccountRelationPool.getRelations(data, dict, feature, relation, ruler);
		UserRelationPool.initRelations(data, dict, ucFeature, uuFeature, ruler, idList);
		
//		System.out.println("Data ready");
/*
		double[] weight = SRW.trainWithoutHer(data, dict, ucFeature, uuFeature,
				relation, ruler, idList);
		
		for (int i = 0; i < weight.length; i++) {
			System.out.println(weight[i] + ",");
		}
*/		
		List<List<List<double[]>>> bbdata = new ArrayList<List<List<double[]>>>();
		List<List<List<Long>>> bbdict = new ArrayList<List<List<Long>>>();
		List<double[][]> bbrelation = new ArrayList<double[][]>();
		List<double[][]> bbucFeature = new ArrayList<double[][]>();
		List<double[][][]> bbuuFeature = new ArrayList<double[][][]>();
		List<int[]> bbruler = new ArrayList<int[]>();
		List<Long[]> bbidList = new ArrayList<Long[]>();
		
		int bball = DataReader.readData(bbdata, bbdict);
//		AccountRelationPool.getRelations(data, dict, feature, relation, ruler);
		UserRelationPool.initRelations(bbdata, bbdict, bbucFeature, bbuuFeature, bbruler, bbidList);
		
		int T = 100;
		double[] weight = SRW.trainSingleDay(relation, ruler, ucFeature,
				uuFeature, alpha, beta, lamda, eta, oRate, tRate, T);
		double[] omega = Arrays.copyOfRange(weight, 0, uuFeature.get(0)[0][0].length);
		double[] theta = Arrays.copyOfRange(weight, uuFeature.get(0)[0][0].length, weight.length);
		
		UserRelationPool.updateRelationMatrix(bbuuFeature, bbruler, omega, bbrelation, eta);
		recordFinalMatrix(bbrelation, bbruler, idList);
		
		LinkerDriver.isRecord = true;
		
		double precision = PageRank.test(bbrelation, bbucFeature, bbruler, theta, alpha);
		
		LinkerDriver.isRecord = false;
		
//		omega = Arrays.copyOfRange(weight, 0, uuFeature.get(0)[0][0].length);
//		theta = Arrays.copyOfRange(weight, uuFeature.get(0)[0][0].length, weight.length);
		
//		UserRelationPool.updateRelationMatrix(bbuuFeature, bbruler, omega, bbrelation, eta);
		
//		double realPrecision = PageRank.test(bbrelation, bbucFeature, bbruler, theta, alpha);
		
//		System.out.println(precision + " vs " + realPrecision);
		
		StringBuilder sb = new StringBuilder();
		sb.append("precision=\t" + precision + "\t");
		sb.append("alpha=\t" + alpha + "\t");
		sb.append("beta=\t" + beta + "\t");
		sb.append("lamda=\t" + lamda + "\t");
		sb.append("eta=\t" + eta + "\t");
		sb.append("oRate=\t" + oRate + "\t");
		sb.append("tRate=\t" + tRate + "\t");
		writer.write(sb.toString() + "\n");
		System.out.print(sb.toString());		
		MyMath.printVector(weight);
	}
	
	public static void recordFinalMatrix(List<double[][]> relation, List<int[]> ruler, List<Long[]> idList) {
		assert (relation.size() == ruler.size());
		String subdir = "matrix\\";
		for (int i = 0; i < relation.size(); i++) {
			MyFileWriter mfw = new MyFileWriter(LinkerDriver.DATA_DIR + subdir + i + "_node.csv", true);
			int[] start = ruler.get(i);
			double[][] matrix = relation.get(i);
			Long[] ids = idList.get(i);
			mfw.write("Id,label,correct\n");
			for (int j = 0; j < start.length - 1; j++) {
				mfw.write(start[j] + "," + ids[start[j]].toString() + "," + 1 + "\n");
				for (int k = start[j] + 1; k < start[j + 1]; k++) {
					mfw.write(k + "," + ids[k].toString() + "," + 0 + "\n");
				}
			}
			mfw.close();
			mfw = new MyFileWriter(LinkerDriver.DATA_DIR + subdir + i + "_edge.csv", true);
			int id = 0;
			mfw.write("Source,Target,Type,Id,Weight\n");
			for (int j = 0; j < matrix.length; j++) {
				for (int k = 0; k < matrix[j].length; k++) {
					if (matrix[j][k] > 0) {
						mfw.write(j + "," + k + "," + "Directed" + "," + id + "," + matrix[j][k] + "\n");
						id++;
					}
				}
			}
			mfw.close();
		}
	}
	
	public static void SupervisedRandomWalks() {
		List<List<List<double[]>>> data = new ArrayList<List<List<double[]>>>();
		List<List<List<Long>>> dict = new ArrayList<List<List<Long>>>();
		List<double[][]> relation = new ArrayList<double[][]>();
		List<double[][]> feature = new ArrayList<double[][]>();
		List<int[]> ruler = new ArrayList<int[]>();

		int all = DataReader.readData(data, dict);
		AccountRelationPool.getRelations(data, dict, feature, relation, ruler);
		// System.out.println(relation.size());
		// System.out.println(feature.size());

		assert (relation.size() == feature.size()) : relation.size();

		System.out.println("Data ready");

		// double[] weight = SRW.train(data, dict);
		double[] weight = SRW.trainWithNeuralModel(data, relation, feature,
				ruler);
		// double[] weight = SRW.trainWithGradient(data, relation, feature,
		// ruler);

		for (int i = 0; i < weight.length; i++) {
			System.out.println(weight[i] + ",");
		}
		PageRank.weight = Arrays.copyOf(weight, weight.length);

	}

	public static void crossValidation(int fold, boolean toTrain) {
		List<List<List<double[]>>> data = new ArrayList<List<List<double[]>>>();
		List<List<List<Long>>> dict = new ArrayList<List<List<Long>>>();
		int all = DataReader.readData(data, dict);
		System.out.println(all);

		List<List<List<double[]>>> train = new ArrayList<List<List<double[]>>>();
		List<List<List<double[]>>> test = new ArrayList<List<List<double[]>>>();
		List<List<List<Long>>> trainDict = new ArrayList<List<List<Long>>>();
		List<List<List<Long>>> testDict = new ArrayList<List<List<Long>>>();

		int listNetAcc = 0, pageRankAcc = 0;

		double[] alphaList = new double[] { 0.05, 0.1, 0.15, 0.2, 0.3, 0.4,
				0.5, 0.6 };
		double[] prP = new double[alphaList.length];
		Arrays.fill(prP, 0.0);

		List<double[][]> relation = new ArrayList<double[][]>();
		List<double[][]> feature = new ArrayList<double[][]>();
		List<int[]> ruler = new ArrayList<int[]>();

		for (int testTime = 0; testTime < fold; testTime++) {

			train.clear();
			test.clear();
			trainDict.clear();
			testDict.clear();

			int line = 0;

			if (fold == 1) {
				Iterator<List<List<double[]>>> iter = data.iterator();
				Iterator<List<List<Long>>> dictIter = dict.iterator();
				while (iter.hasNext() && dictIter.hasNext()) {
					List<List<double[]>> d = iter.next();
					List<List<Long>> u = dictIter.next();
					test.add(d);
					testDict.add(u);
					train.add(d);
					trainDict.add(u);
				}
			} else {
				Iterator<List<List<double[]>>> iter = data.iterator();
				Iterator<List<List<Long>>> dictIter = dict.iterator();
				while (iter.hasNext() && dictIter.hasNext()) {
					if (line % fold == testTime) {
						test.add(iter.next());
						testDict.add(dictIter.next());
					} else {
						train.add(iter.next());
						trainDict.add(dictIter.next());
					}
					line++;
				}
			}

			feature.clear();
			relation.clear();
			ruler.clear();
			AccountRelationPool.getRelations(test, testDict, feature, relation,
					ruler);

			if (toTrain) {
				
				double[] weight = ListNet.train(train);
				LinkerDriver.isRecord = true;
				listNetAcc += ListNet.test(test, weight);
				LinkerDriver.isRecord = false;
				for (int i = 0; i < weight.length; i++) {
					System.out.print(weight[i] + "\t");
				}
				System.out.println("");
				PageRank.weight = Arrays.copyOf(weight, weight.length);
				// pageRankAcc += PageRank.test(test, testDict);
				/*
				for (int i = 0; i < alphaList.length; i++) {
					prP[i] += PageRank.test(relation, feature, ruler,
							PageRank.weight, alphaList[i]);
				}
				*/

			} else {
				listNetAcc += ListNet.test(test, PageRank.weight);
				// pageRankAcc += PageRank.test(test, testDict);
				for (int i = 0; i < alphaList.length; i++) {
					prP[i] += PageRank.test(relation, feature, ruler,
							PageRank.weight, alphaList[i]);
				}
			}

			// for (PageRank.alpha = 0.0; PageRank.alpha < 1; PageRank.alpha +=
			// 0.05) {
			// System.out.println("alpha = " + PageRank.alpha);
			// PageRank.test(data, dict);
			// }

		}

		System.out.println("ListNet:  " + listNetAcc + "/" + all + " = "
				+ ((double) listNetAcc / all));
		// System.out.println("PageRank: " + pageRankAcc + "/" + all + " = "
		// + ((double) pageRankAcc / all));
		/*
		System.out.println("PageRank: ");
		for (int i = 0; i < alphaList.length; i++) {
			prP[i] /= fold;
		}
		double bestAlpha = 0.0;
		double bestPre = 0.0;
		for (int i = 0; i < alphaList.length; i++) {
			if (bestPre < prP[i]) {
				bestPre = prP[i];
				bestAlpha = alphaList[i];
			}
			System.out.println("alpha = " + alphaList[i] + "\t" + (prP[i]));
		}
		System.out.println("Best alpha = " + bestAlpha + " with precision "
				+ bestPre);
		 */
	}

	public static void srw(List<QueryContext> contextList, AccountPool ap) {
		Iterator<QueryContext> qcIter = contextList.iterator();
		int acc = 0, all = 0;
		while (qcIter.hasNext()) {
			QueryContext context = qcIter.next();
			// context.washQuery();
			RefGraph rg = new RefGraph(qcIter.next(), ap);
			acc += rg.predict();
			all += context.queryList.size();
		}
		System.out.println("Got " + acc + " correct out of " + all
				+ " with precision " + ((double) acc / all));
	}

	public static void test(int fold, List<QueryContext> contextList) {
		int acc = 0;
		int all = 0;
		for (int critera = 0; critera < fold; critera++) {
			List<QueryContext> train = new ArrayList<QueryContext>();
			List<QueryContext> test = new ArrayList<QueryContext>();
			Iterator<QueryContext> iter = contextList.iterator();
			int counter = 0;
			while (iter.hasNext()) {
				if (counter % fold == critera) {
					QueryContext qc = iter.next();
					test.add(qc);
					for (int i = 0; i < qc.queryList.size(); i++) {
						if (qc.queryList.get(i).isLinked) {
							all++;
						}
					}

				} else {
					train.add(iter.next());
				}
				counter++;
			}

			acc += ListwiseTrainDriver.test(train, test);
		}
		System.out.println(fold + " - fold test:");
		System.out.println("\t Got " + acc + " correct out of " + all
				+ " with precision " + ((double) acc / all));
	}

}
